{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "3bd8feed-460d-461c-94b0-5f65901d150e",
   "metadata": {},
   "source": [
    "## Importing Required Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9ff37662-e5f6-4e24-85de-b001a671bc84",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "from transformers import BertTokenizer\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "sns.set(style=\"darkgrid\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "508411c5-461d-4bb0-928b-e43ab774e4fb",
   "metadata": {},
   "source": [
    "## Importing the Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "9bc5f8b0-0bc4-4150-bd37-0c5d2b9dd6f9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(14675, 2)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentiment</th>\n",
       "      <th>review</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Good but need updates and improvements</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>Worst mobile i have bought ever, Battery is dr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>when I will get my 10% cash back.... its alrea...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>The worst phone everThey have changed the last...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sentiment                                             review\n",
       "0          1             Good but need updates and improvements\n",
       "1          0  Worst mobile i have bought ever, Battery is dr...\n",
       "2          1  when I will get my 10% cash back.... its alrea...\n",
       "3          1                                               Good\n",
       "4          0  The worst phone everThey have changed the last..."
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file_path = r\"D:\\\\Paper Dataset\\\\K8 Reviews v0.2.csv\"\n",
    "df = pd.read_csv(file_path)\n",
    "\n",
    "# Display the first few rows of the DataFrame\n",
    "print(df.shape)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "8a5dcc35-2e81-4ef4-b0bc-78b225f2ea58",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['sentiment'] = df['sentiment'].replace({1: 'positive', 0: 'negative'})\n",
    "df['sentiment'] = df['sentiment'].astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "c6d4472d-e5a8-4564-82db-c26228bc91f8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "sentiment\n",
       "negative    7712\n",
       "positive    6963\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"sentiment\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "cf652a10-e105-41a6-9c96-45919ebe9eef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkcAAAHuCAYAAACVuKhMAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAABSw0lEQVR4nO3deVhV5d7/8TczCKKAClpahjngPIB6TjhV5kmz1E4+OeSsOWRqznqcEvMxU3LANHHOo5ZomZZpZTYoilpWiuUsKaKCoAIie+/fH/7YzyIpBoGN8Hldl5fste517+9m7wUf7nWvtewsFosFEREREQHA3tYFiIiIiBQlCkciIiIiBgpHIiIiIgYKRyIiIiIGCkciIiIiBgpHIiIiIgYKRyIiIiIGCkciIiIiBgpHIlKgivt1Zm39+mz9/CLFkcKRiADQs2dPatSokelfzZo1adSoEZ07d+bjjz/OdZ+HDh1i4MCB1scxMTHUqFGDiIiI/Cw9SxnPZfxXp04d/vGPfzB48GAOHz6cqX1kZCQ1atQgMjIyR/2npaUxa9Ystm3blm3bGjVqsHDhwjw9z9+x5fdXpDhztHUBIlJ0BAQEMHXqVOtjk8lEbGwsq1atYuzYsZQtW5aWLVvmuL8PP/yQU6dOWR9XqFCBjRs3UqVKlXyt++8MHjyYVq1aAXD79m1iY2NZu3Yt3bt3Z+HChTz11FMA1K5dm40bN1KtWrUc9RsXF8fq1at56623sm27ceNG/Pz88vwa/kpR+P6KFEcKRyJi5eHhQYMGDe5Z3qJFC5o3b05ERESuwtGfOTs7Z9l/QapSpco9z/mvf/2LHj16MGnSJJo1a4aHh8dfvvb8UFiv2RbfX5HiSIfVRCRbLi4uODs7Y2dnZ10WHx/P9OnTad26NXXq1CEoKIihQ4cSExMDwPjx49myZQt//PGH9VDPnw/7REREEBAQwE8//UTXrl2pW7curVu3Jjw8PNPzx8XFMXLkSIKCgggMDGTKlCnMnz+fNm3a5On1ODs789prr3H9+nU+++wz4N7DXampqUybNo0WLVpQp04d2rVrZ60rJiaGJ598EoAJEyZY6xg/fjy9evVi6tSpNGrUiGeffRaTyZTpsFqGkydP0q1bN+rWrcvTTz/N2rVrrev+6vDY+PHjMz1Xdt9fgLNnzzJ8+HD++c9/0qBBA3r27MmhQ4fuea7PPvuM4cOH07BhQ4KCgpg8eTLJycl5+v6KPOg0ciQiVhaLhfT0dOtjk8nEH3/8weLFi7l16xbPP/+8td2gQYNITExk9OjRlCtXjhMnThAaGsrUqVMJDw9nyJAhxMfHc+zYMRYtWkSVKlWy/GVrNpsZMWIEvXv3ZsSIEXz00UfMmTOH6tWrExwcTFpaGr169SI5OZmJEyfi4eHBsmXLOH78OOXLl8/za23evDn29vYcPnyYf//73/esnzVrFt999x3jxo2jXLly7N27lzlz5lC2bFmee+45Fi1axLBhwxg8eDBt27a1bhcVFYWLiwuLFy8mOTkZBweHLJ//rbfe4pVXXmHIkCF89dVXzJw5E7PZTK9evXJUf06+vydPnuSll17i0UcfZfLkyTg5ObFmzRp69erFihUrCAoKsradOnUqXbp0ISwsjKNHjzJ//ny8vLx44403clSPSHGicCQiVgcPHqR27dqZltnZ2VG9enXeffddWrduDdwdyXFzc2PcuHE0adIEgKZNm3L+/Hk2btwI3D2c5e3tnelQT1bhyGKxMGTIEGtAady4Mbt27WLPnj0EBwfzySefcPr0aTZv3kydOnUAaNasmXWuUF45Ojri5eXFlStXslx/4MAB/vnPf9K+fXvr6ytVqhQ+Pj44OztTq1Yt6+sMCAiwbpeens6MGTOynWP00ksvMXbsWACeeOIJLl++zNKlS+nZs2eO6s/J93fRokU4OzuzZs0aPDw8AGjVqhUdOnRgzpw5fPTRR9a2LVu2ZNy4ccDd4Pj999+zZ88ehSMpkRSORMSqdu3aTJ8+HbgbgEJDQ7lz5w6hoaE89thj1na+vr6sWbMGi8VCTEwM586d4/Tp0xw+fJi0tLRcP2/Dhg2tXzs7O+Pt7W39Rb9//34qV65sDUZwd25U69at7/uML4vFkulQoVHTpk3ZsGEDsbGxtGzZkpYtWzJ06NBs+yxbtmyOJl8/++yzmR4//fTT7N69m9OnT+Pq6pqzF5CNAwcO0Lp1a2swgruhsH379tbRwAx/nqvk5+fHH3/8kS91iDxoFI5ExMrd3Z26detaH9evX5+OHTvSt29fIiIi8Pb2tq775JNPmDdvHpcuXaJs2bLUqlUrz7/U/7ydvb299fo9CQkJ+Pj43LNNVstyIyUlhcTExL8MMpMmTcLPz49PPvmEN998kzfffJOGDRsybdo0atas+Zf9uru75+j5y5Url+lxxutJTEzMt3CUmJh4z/NkPLfFYuHmzZvWZW5ubpnaGN8DkZJGE7JF5C+VK1eOKVOmcOnSJUJCQqzLo6KiGDduHG3btmXv3r1ERkayatWqAjlTytfXl6tXr96z/Nq1a/fV74EDBzCZTAQGBma53tnZmcGDB/PZZ5/x9ddfM2XKFC5cuJBvh5kSExMzPc54jT4+PtbRLJPJlKlNbidIlylTJsvvXcahRC8vr1z1J1JSKByJyN9q164dwcHBfPrppxw4cACAI0eOYDabee211/D19QXu/iL/4YcfgLuTrOHu6MP9CgoKIiYmhuPHj1uXpaam8u233+a5z/T0dMLCwihXrhxPP/30PetTU1N55plnWLFiBQCVKlWie/futG/fnosXLwL85UTrnNqzZ0+mx9u3b6dixYo88sgj1sNgly9ftq6/c+cOR48ezbRNdt/fwMBAvv7660wjRCaTie3bt1O3bl2cnZ3v6zWIFFc6rCYi2Zo4cSIdO3Zk5syZbNmyhXr16gEwY8YMunTpQmJiIh988AHR0dHA3REODw8PPD09uXr1Kt988411AnNudejQgWXLljF06FBef/11PD09WblyJdeuXaNSpUrZbn/+/Hl+/PFH4G7AiImJYcOGDfz6668sXrz4nsNJcPcwX+3atVm0aBFOTk7UqFGDM2fOsGXLFp555hkASpcuDcC+ffvw9/enfv36uXpda9euxd3dnYCAALZv3863337LnDlzsLOzo0yZMjRs2JC1a9fyyCOPUKZMGdasWUNqaiqlSpWy9pHd93fYsGHs3buXV155hYEDB+Lk5MS6deu4cOECy5cvz1W9IiWJRo5EJFuPPfYYPXv25MSJE/z3v/+ladOmTJkyhSNHjjBgwABmz55NpUqVWLRoEYD1OjqdO3fmoYceYujQoWzdujVPz+3o6Eh4eDgBAQFMmzaNsWPH8vjjj/P0009nCgp/ZcmSJXTt2pWuXbvSu3dv5s2bR6VKldi4cePfXtByxowZdO7cmRUrVtC3b1/CwsJ48cUXmTZtGnB3UnifPn3YvXs3AwYM4M6dO7l6XTNnzuTzzz9n4MCBHD58mHnz5lkvlQAwe/Zs6tSpw+TJk5kwYQK1a9e+5zT/7L6/jz/+OOvXr8fHx4cJEyYwZswYLBYLa9as4R//+Eeu6hUpSewsmnEnIkXY77//zunTp2nbtm2mM8tefPFF/Pz8rIFMRCS/6LCaiBRpycnJvP7663Tr1o2nn34ak8nEjh07+OWXXxg9erStyxORYkgjRyJS5H3++eeEh4dz6tQpLBYLAQEBDB48mCeeeMLWpYlIMaRwJCIiImKgCdkiIiIiBgpHIiIiIgYKRyIiIiIGNg9H6enp1rt9N2zYkO7du1sv2AZw/PhxevToQYMGDWjTpg1r1qzJtL3ZbGbBggUEBwfToEEDBgwYwIULFzK1ya4PERERkQw2P5V/yZIlfPjhh8yePZvKlSvz/vvv079/f3bs2IGTkxN9+vShTZs2TJ8+nR9//JHp06fj7u5Oly5dAAgLC2P9+vXMnj0bPz8/3n77bfr378+2bdtwdnYmISEh2z7ywmKxYDZrLntJYW9vp/dbpJjS/l1y2NvbZbpe2l+x+dlqzz//PM2bN2f8+PEA3Lx5k8aNG7Nw4ULOnDnDunXr+Prrr3F0vJvj5s2bx86dO9m5cydpaWk0a9aM0aNH061bNwCSkpIIDg4mJCSEDh06sHTp0r/tI69MJjPx8bfu89XLg8DR0R4vL3cSEm6Rnm62dTkiko+0f5cs3t7uODhkf9DM5ofVfHx8+Prrr4mJicFkMrFx40acnZ2pWbMmUVFRBAUFWUMNQLNmzTh79ixXr14lOjqaW7du0bx5c+t6T09PAgICOHjwIEC2fYiIiIgY2fyw2qRJk3j99dd58skncXBwwN7enoULF1KlShViY2OpXr16pvYVKlQA4NKlS8TGxgJQsWLFe9pkrMuuj3LlyuW5dkdHm2dLKQQZf2Xk5K8NEXmwaP+WrNg8HJ08eZLSpUuzePFifH19+fDDDxk9ejTr1q0jNTUVZ2fnTO1dXFwAuH37NikpKQBZtklMTATIto+8sre3w8vLPc/by4PH0/Peu7eLSPGg/VuMbBqOLl26xBtvvMGqVato0qQJAHXr1uXkyZMsXLgQV1dX0tLSMm2TEWhKlSqFq6srAGlpadavM9q4ud39oGfXR16ZzRaSkpLzvL08OBwc7PH0dCMpKQWTSXMSRIoT7d8li6enW45GCW0ajn766Sfu3LlD3bp1My2vX78+e/fupVKlSsTFxWVal/HY19eX9PR067IqVapkalOjRg0A/Pz8/raP+6HJeyWLyWTWey5STGn/FiObHmT18/MD4MSJE5mW//bbbzz66KMEBgZy6NAhTCaTdd3+/fupWrUqPj4+1KxZEw8PDyIjI63rk5KSOHbsGIGBgQDZ9iEiIiJiZNNwVK9ePRo3bsy4cePYv38/Z8+eJTQ0lH379jFw4EC6dOnCzZs3mTRpEidPniQiIoJVq1YxaNAg4O5cox49ejB37ly+/PJLoqOjGTlyJH5+frRt2xYg2z5EREREjGx+naPExERCQ0PZs2cPiYmJVK9enVGjRhEUFATA0aNHCQkJ4dixY5QvX56+ffvSo0cP6/Ymk4l58+YRERFBamoqgYGBTJkyhYcfftjaJrs+8kLXOSo5dB0UkeJL+3fJktPrHNk8HD2oFI5KDv3wFCm+tH+XLA/MRSBFREREihKFIxEREREDhSMRERERA4UjEREREQOFIxEREREDhSMRERERA5vfeFYeLPb2dtjb29m6jEJVku/abTZbMJt1tQ8RKVkUjiTH7O3tKFu2VIkMCVAy79ptMpm5fj1ZAUlEShSFI8kxe3s7HBzsmfvBIWIu37B1OVLAHvYtzejujbG3t1M4EpESReFIci3m8g1O/ZFo6zJEREQKRMk8PiIiIiLyFxSORERERAwUjkREREQMFI5EREREDBSORERERAwUjkREREQMFI5EREREDBSORERERAwUjkREREQMFI5EREREDBSORERERAwUjkREREQMFI5EREREDBSORERERAwUjkREREQMFI5EREREDBSORERERAwUjkREREQMFI5EREREDBSORERERAwUjkREREQMFI5EREREDBSORERERAwUjkREREQMbBqOIiMjqVGjRpb/nnzySQBiYmIYNGgQjRo14oknniA0NBSTyZSpnw8++IAnn3ySevXq0a1bN44dO5ZpfU76EBEREQFwtOWTN2zYkO+++y7Tsh9//JHXXnuNIUOGcOfOHfr168ejjz7Khg0bOH/+PJMmTcLe3p7hw4cDsGXLFubMmcObb75JQEAAy5Yto0+fPnz22Wd4e3vnqA8RERGRDDYNR87OzpQvX976ODk5mbfeeotOnTrRpUsXPv30Uy5evMimTZsoU6YM1atX59q1a8yZM4dXX30VZ2dn3nvvPXr06EHHjh0BmDVrFk899RQffvghgwYNYufOndn2ISIiIpKhSM05eu+990hJSWHcuHEAREVFUbt2bcqUKWNt06xZM27evMnx48e5du0aZ8+epXnz5tb1jo6ONGnShIMHD+aoDxEREREjm44cGcXHx7Nq1SreeOMNypYtC0BsbCx+fn6Z2lWoUAGAS5cu4eh4t/yKFSve0yY6OjpHfdSvXz/PNTs6FqlsWeAcHErW65W79L5LcZbx+dbnXIyKTDhav349pUuXpmvXrtZlqampeHp6Zmrn4uICwO3bt0lJSQG459CYi4sLt2/fzlEfeWVvb4eXl3uetxd5UHh6utm6BJECp8+5GBWZcLR161ZeeOEFXF1drctcXV1JS0vL1C4j0JQqVcraNqs2bm5uOeojr8xmC0lJyXne/kHk4GCvHyAlUFJSCiaT2dZlSCGws7PD3t7O1mUUKnt7Ozw8XLl5MxWz2WLrcgqV2WzBYilZr9nT0y1Ho4RFIhxFR0dz4cIFnnvuuUzL/fz8+O233zIti4uLA8DX19d6OC0uLg5/f/9MbXx9fXPUx/1IT9cvDCn+TCazPuslgL29HWXL5uwXR3Hk4eGafaNixmQyc/16cokLhTlRJMJRVFQUPj4+1KxZM9PywMBAtm7dys2bN/Hw8ABg//79uLu7U7NmTZydnalatSqRkZHWSdnp6elERUXRrVu3HPUhIiJ3w5GDgz1zPzhEzOUbti5HCtjDvqUZ3b0x9vZ2CkdZKBLh6NixY9SoUeOe5U899RShoaGMGDGC0aNHExMTw7x58+jbt691nlHfvn0JCQnhkUceoW7duixbtozU1FRefPHFHPchIiJ3xVy+wak/Em1dhohNFYlwdOXKFesZakYuLi4sX76c6dOn89JLL1GmTBm6devGkCFDrG1eeuklbty4QWhoKNevX6dOnTqsXLkSb2/vHPchIiIikqFIhKP333//L9c98sgjrFix4m+379evH/369buvPkRERESgiF0EUkRERMTWFI5EREREDBSORERERAwUjkREREQMFI5EREREDBSORERERAwUjkREREQMFI5EREREDBSORERERAwUjkREREQMFI5EREREDBSORERERAwUjkREREQMFI5EREREDBSORERERAwUjkREREQMFI5EREREDBSORERERAwUjkREREQMFI5EREREDBSORERERAwUjkREREQMFI5EREREDBSORERERAwUjkREREQMFI5EREREDBSORERERAwUjkREREQMFI5EREREDBSORERERAwUjkREREQMFI5EREREDBSORERERAwUjkREREQMikQ42rp1K88++yx169alffv2fPbZZ9Z1MTExDBo0iEaNGvHEE08QGhqKyWTKtP0HH3zAk08+Sb169ejWrRvHjh3LtD4nfYiIiIhAEQhHH3/8MZMmTaJ79+5s376dDh06MGrUKI4cOcKdO3fo168fABs2bGDatGn897//ZfHixdbtt2zZwpw5c3j99deJiIjg4Ycfpk+fPsTHxwPkqA8RERGRDI62fHKLxcK7777LK6+8Qvfu3QEYPHgwUVFRHDhwgD/++IOLFy+yadMmypQpQ/Xq1bl27Rpz5szh1VdfxdnZmffee48ePXrQsWNHAGbNmsVTTz3Fhx9+yKBBg9i5c2e2fYiIiIhksOnI0ZkzZ/jjjz947rnnMi0PDw9n0KBBREVFUbt2bcqUKWNd16xZM27evMnx48e5du0aZ8+epXnz5tb1jo6ONGnShIMHDwJk24eIiIiIkU1Hjs6cOQNAcnIy/fr149ixYzz88MMMHjyYNm3aEBsbi5+fX6ZtKlSoAMClS5dwdLxbfsWKFe9pEx0dDZBtH/Xr189z/Y6ONj8qWagcHErW65W79L6XDHqfSya971mzaTi6efMmAOPGjWPYsGGMHj2anTt3MmTIEFauXElqaiqenp6ZtnFxcQHg9u3bpKSkANxzaMzFxYXbt28DZNtHXtnb2+Hl5Z7n7UUeFJ6ebrYuQUQKiPbvrNk0HDk5OQHQr18/OnXqBECtWrU4duwYK1euxNXVlbS0tEzbZASaUqVK4erqCpBlGze3u294dn3kldlsISkpOc/bP4gcHOy1I5VASUkpmExmW5chBUz7d8lU0vZvT0+3HI2W2TQc+fr6AlC9evVMy6tVq8aePXsICgrit99+y7QuLi7Oum3G4bS4uDj8/f0ztcno28/P72/7uB/p6SXnAyUll8lk1mddpJjS/p01mx5srF27Nu7u7vz000+Zlv/2229UqVKFwMBAjh07Zj38BrB//37c3d2pWbMmPj4+VK1alcjISOv69PR0oqKiCAwMBMi2DxEREREjm4YjV1dX+vfvz+LFi/n00085f/48S5Ys4fvvv6dPnz489dRTlC9fnhEjRhAdHc3u3buZN28effv2tc4z6tu3LytXrmTLli2cPHmSiRMnkpqayosvvgiQoz5EREREMtj0sBrAkCFDcHNzY/78+Vy+fBl/f38WLlxI06ZNAVi+fDnTp0/npZdeokyZMnTr1o0hQ4ZYt3/ppZe4ceMGoaGhXL9+nTp16rBy5Uq8vb2Bu5Ovs+tDREREJIOdxWKx2LqIB5HJZCY+/patyyhUjo72eHm5M2LeHk79kWjrcqSA+T9UhtBRrUhIuKU5CSWA9u+SpaTu397e7jmakK0LHIiIiIgYKByJiIiIGCgciYiIiBgoHImIiIgYKByJiIiIGCgciYiIiBgoHImIiIgYKByJiIiIGCgciYiIiBgoHImIiIgYKByJiIiIGCgciYiIiBgoHImIiIgYKByJiIiIGCgciYiIiBgoHImIiIgYKByJiIiIGCgciYiIiBgoHImIiIgYKByJiIiIGCgciYiIiBgoHImIiIgYKByJiIiIGCgciYiIiBgoHImIiIgYKByJiIiIGCgciYiIiBjkSzhKT0/n+vXr+dGViIiIiE3lOhylp6ezaNEitm3bBkBkZCT//Oc/ad68Ob169SIxMTHfixQREREpLLkORwsWLGDJkiUkJSUBMHPmTMqWLcuECRM4f/4877zzTr4XKSIiIlJYch2Otm/fzqhRo+jevTunTp3i999/Z/DgwbzyyiuMHDmSr776qiDqFBERESkUuQ5HcXFx1K9fH4A9e/Zgb29PixYtAPDz8+PGjRv5W6GIiIhIIcp1OKpQoQIxMTEAfPXVV9SqVQtvb28Ajhw5gp+fX/5WKCIiIlKIch2OOnTowFtvvUW/fv04dOgQXbp0ASAkJISFCxfy3HPP5aq/y5cvU6NGjXv+RUREAHD8+HF69OhBgwYNaNOmDWvWrMm0vdlsZsGCBQQHB9OgQQMGDBjAhQsXMrXJrg8RERGRDI653WDEiBGUKlWKgwcP8sYbb9CtWzcAfv75Z/r27cuQIUNy1V90dDQuLi7s3r0bOzs76/LSpUuTkJBAnz59aNOmDdOnT+fHH39k+vTpuLu7W0NZWFgY69evZ/bs2fj5+fH222/Tv39/tm3bhrOzc476EBEREcmQ63B09OhRBg0axKBBgzIt37BhQ54K+O2333j00UepUKHCPetWr16Nk5MTM2bMwNHREX9/f86dO8eyZcvo0qULaWlprFixgtGjR9OqVSsA5s+fT3BwMF988QUdOnRg06ZNf9uHiIiIiFGuD6t17dqVf/7zn0ycOJFdu3aRnJx8XwWcOHECf3//LNdFRUURFBSEo+P/ZbhmzZpx9uxZrl69SnR0NLdu3aJ58+bW9Z6engQEBHDw4MEc9SEiIiJilOuRoy1btrB3716+++47Ro4cib29PYGBgbRu3ZpWrVrx8MMP56q/3377DS8vL7p3786ZM2d45JFHGDx4MC1atCA2Npbq1atnap8xwnTp0iViY2MBqFix4j1tMtZl10e5cuVyVa+Ro2PJuvuKg0PJer1yl973kkHvc8mk9z1ruQ5HtWrVolatWgwaNIibN2+yb98+9u7dS3h4OCEhIVSrVs169ezspKenc/r0aapVq8b48ePx8PBg+/btDBw4kJUrV5Kamoqzs3OmbVxcXAC4ffs2KSkpAFm2ybhSd3Z95JW9vR1eXu553l7kQeHp6WbrEkSkgGj/zlquw5HRjRs3SE5OxmQyYW9vj8VisY7Y5OjJHR2JjIzEwcEBV1dXAOrUqcPvv/9OeHg4rq6upKWlZdomI9CUKlXKuk1aWpr164w2bm533/Ds+sgrs9lCUtL9HVJ80Dg42GtHKoGSklIwmcy2LkMKmPbvkqmk7d+enm45Gi3LdTjavHkzUVFRHDhwgD/++AM3NzcaNWrE//zP/9C0aVPq1KmTq/7c3e8dfXn88cf57rvv8PPzIy4uLtO6jMe+vr6kp6dbl1WpUiVTmxo1agBk28f9SE8vOR8oKblMJrM+6yLFlPbvrOU6HE2aNAk7Oztq167NhAkTaNWqVabJzrnx+++/07VrV5YsWULTpk2ty3/55ReqVatGrVq12LBhAyaTCQcHBwD2799P1apV8fHxoXTp0nh4eBAZGWkNR0lJSRw7dowePXoAEBgY+Ld9iIiIiBjleibW7NmzeeGFF7h69SqvvfYaHTt2ZNq0aezYsYNr167lqi9/f38ee+wxZsyYQVRUFKdOneKtt97ixx9/ZPDgwXTp0oWbN28yadIkTp48SUREBKtWrbJeRsDZ2ZkePXowd+5cvvzyS6Kjoxk5ciR+fn60bdsWINs+RERERIxyPeTzwgsv8MILLwBw5swZ9u/fz/79+5k1axbXrl3jscceY/v27Tnqy97envfee4933nmHESNGkJSUREBAACtXrrSeYbZ8+XJCQkLo1KkT5cuXZ+zYsXTq1Mnax/Dhw0lPT2fy5MmkpqYSGBhIeHg4Tk5OAPj4+GTbh4iIiEiG+5qQ/fDDD+Pv78/ly5e5fPkyV69e5cqVK7nqo1y5crz11lt/ub5evXps3LjxL9c7ODgwZswYxowZk+c+RERERDLkOhxFR0fzww8/8MMPP3Do0CFSU1OpWrUqLVu2ZOTIkTRu3Lgg6hQREREpFHk6rObs7ExQUBBvvPEGLVu2pHLlygVRm4iIiEihy3U4CgsLo3nz5tbrCImIiIgUJ7kOR23atCEtLY3169fzww8/cOXKFWbNmsWBAweoXbs29erVK4g6RURERApFrk/lj4+Pp0uXLoSEhHDu3DmOHj1Kamoqe/bsoWfPnhw5cqQg6hQREREpFLkOR3PmzOHWrVvs2LGDLVu2YLFYAFiwYAF169ZlwYIF+V6kiIiISGHJdTj6+uuvef3113nkkUews7OzLndxcaFv3778+uuv+VqgiIiISGHKdTi6ffs2ZcuWzXKdg4MDd+7cud+aRERERGwm1+Gobt26rF+/Pst127Zty/WNZ0VERESKklyfrfb666/Tu3dvnn/+eVq2bImdnR2ffvopCxcu5LvvvmP58uUFUaeIiIhIocj1yFGTJk1YuXIlbm5uLF++HIvFwqpVq7hy5QpLly6lWbNmBVGniIiISKHI073VAgMD2bBhA6mpqSQmJuLh4YG7u3t+1yYiIiJS6HIUji5evEj58uVxcnLi4sWL96xPTEwkMTHR+rhSpUr5V6GIiIhIIcpROHryySfZuHEj9erVo02bNplO4c/K8ePH86U4ERERkcKWo3A0a9Ys681lZ82alW04EhEREXlQ5SgcderUyfp1q1at8Pb2LrCCRERERGwp12ertWjRgsGDB/P555+TlpZWEDWJiIiI2Eyuw9Ho0aO5du0aI0aM4J///CeTJ08mKiqqIGoTERERKXS5PpW/d+/e9O7dmwsXLvDpp5+yY8cOPvroIypVqkTHjh157rnn8Pf3L4haRURERApcrkeOMlSuXJnBgwezbds2tm3bRqtWrXj//ffp0KFDftYnIiIiUqjydBHIDNeuXeOzzz7js88+48iRI5QtW5Znn302v2oTERERKXS5Dkc3btxg586dbN++nYMHD+Lg4ECbNm0ICwsjODgYBweHgqhTREREpFDkOhw1b94cs9lM48aNmTZtGu3atcPDw6MgahMREREpdLkOR6+99hrPPfecbhEiIiIixVKuw9GgQYOAu/dTi4qKIi4ujmeeeYbr169TtWpVXT1bREREHmh5mpC9ZMkSli5dSmpqKnZ2dtSrV4/Q0FASEhJYsWIFnp6e+V2niIiISKHI9an869atY+HChfTp04dNmzZhsVgA6NGjBxcuXODdd9/N9yJFRERECkuuw9HatWsZOHAgr7/+OrVr17Yub9myJSNGjOCrr77K1wJFREREClOuw9HFixcJCgrKct1jjz3G1atX77soEREREVvJdTiqWLEiR44cyXLdL7/8QsWKFe+7KBERERFbyfWE7BdffJGFCxfi6upKq1atAEhOTmbnzp0sXbqUPn365HeNIiIiIoUm1+FowIABxMTEMHfuXObOnQvAK6+8AsBzzz1nPdVfRERE5EGU63BkZ2fHjBkz6Nu3L/v37+f69euULl2awMBAqlevXhA1ioiIiBSaPN949tFHH+XRRx/NtMxisbB+/Xq6d+9+v3WJiIiI2ESOJ2Tv3buXkSNHMmrUKL755pt71kdFRdGpUydmzpyZ52LOnDlDw4YNiYiIsC47fvw4PXr0oEGDBrRp04Y1a9Zk2sZsNrNgwQKCg4Np0KABAwYM4MKFC5naZNeHiIiISIYchaNPPvmEgQMHsnv3br755hteffVVdu3aBcD169cZPXo0PXv25OTJk3mekH3nzh1Gjx5NcnKydVlCQgJ9+vShSpUqbN68maFDhzJ37lw2b95sbRMWFsb69et588032bBhA2azmf79+5OWlpbjPkREREQy5Oiw2urVq6lfvz7h4eE4OzszYcIEFi9ezOOPP06fPn24dOkSwcHBTJw4kapVq+apkIULF+Lh4ZFp2aZNm3BycmLGjBk4Ojri7+/PuXPnWLZsGV26dCEtLY0VK1YwevRo65lz8+fPJzg4mC+++IIOHTpk24eIiIiIUY5Gjs6ePUuvXr3w8PDA2dmZYcOGceLECYYMGUJaWhrvvvsu77//fp6D0cGDB9m4cSOzZ8/OtDwqKoqgoCAcHf8vwzVr1oyzZ89y9epVoqOjuXXrFs2bN7eu9/T0JCAggIMHD+aoDxERERGjHI0cJScnZ7q440MPPYTFYsHR0ZFPPvkEHx+fPBeQlJTE2LFjmTx58j0XkIyNjb3nDLgKFSoAcOnSJWJjYwHu2a5ChQrWddn1Ua5cuTzX7uiY62toPtAcHErW65W79L6XDHqfSya971nLUTiyWCw4ODhYH2d8PXLkyPsKRgDTpk2jYcOGPPfcc/esS01NxdnZOdMyFxcXAG7fvk1KSgpAlm0SExNz1Ede2dvb4eXlnuftRR4Unp5uti5BRAqI9u+s5flUfvi/EZi82rp1K1FRUWzbti3L9a6urtaJ1RkyAk2pUqVwdXUFIC0tzfp1Rhs3N7cc9ZFXZrOFpKTk7BsWIw4O9tqRSqCkpBRMJrOty5ACpv27ZCpp+7enp1uORsvuKxzZ2dndz+Zs3ryZa9euWSdTZ5g6dSo7duzAz8+PuLi4TOsyHvv6+pKenm5dVqVKlUxtatSoAZBtH/cjPb3kfKCk5DKZzPqsixRT2r+zluNwNG3aNOvZZBaLBYD//Oc/uLtnPrRkZ2fH6tWrc9Tn3LlzSU1NzbSsbdu2DB8+nI4dO/Lxxx+zYcMGTCaT9VDe/v37qVq1Kj4+PpQuXRoPDw8iIyOt4SgpKYljx47Ro0cPAAIDA/+2DxERERGjHM3ECgwMxN3dHYvFYg1GgYGBlCpVyros45/ZnPME6uvryyOPPJLpH4CPjw++vr506dKFmzdvMmnSJE6ePElERASrVq2y3r/N2dmZHj16MHfuXL788kuio6MZOXIkfn5+tG3bFiDbPkRERESMcjRytHbt2oKuI0s+Pj4sX76ckJAQOnXqRPny5Rk7diydOnWythk+fDjp6elMnjyZ1NRUAgMDCQ8Px8nJKcd9iIiIiGS4rzlHBeHEiROZHterV4+NGzf+ZXsHBwfGjBnDmDFj/rJNdn2IiIiIZNAFDkREREQMFI5EREREDBSORERERAxyFI4GDhzI77//Dty9D9qtW7cKtCgRERERW8lRONq3bx/Xrl0D4JVXXuHUqVMFWpSIiIiIreTobLVKlSoxdepUGjVqhMViISwsDC8vryzb2tnZMWvWrHwtUkRERKSw5CgczZgxgzlz5nDgwAHs7Oz45Zdf7rmZa4b7vaWIiIiIiC3lKBw1bdqUzZs3A1CzZk3CwsKoV69egRYmIiIiYgu5vgjkl19+SYUKFQBISUnh5s2blC1b1npFahEREZEHWa7D0UMPPURUVBRz5szhl19+sd5rrV69eowcOZJmzZrle5EiIiIihSXX4ejw4cP07t2bypUrM2TIEMqVK0dcXBzbt2+nf//+rF27loYNGxZErSIiIiIFLtfhKDQ0lCZNmhAeHo6Dg4N1+bBhw+jXrx8LFy5kxYoV+VqkiIiISGHJ9RWyf/75Z1555ZVMwQjA3t6eHj16cPTo0XwrTkRERKSw5Tocubu7k56enuW69PR06xwkERERkQdRrsNRo0aNWLZsGSkpKZmWJycns2zZMpo0aZJvxYmIiIgUtlzPOXrjjTfo3LkzTz75JK1ataJ8+fJcuXKFPXv2kJqaSkhISEHUKSIiIlIoch2OHnnkETZt2sTChQv55ptvSExMpEyZMgQFBTFs2DCqVatWEHWKiIiIFIpchyMAf39/QkND87kUEREREdvL9ZwjERERkeJM4UhERETEQOFIRERExEDhSERERMRA4UhERETEIF/DUXx8PFu3bs3PLkVEREQKVb6GowsXLjBhwoT87FJERESkUOVrOPL392f16tX52aWIiIhIocrXcOTh4UFQUFB+dikiIiJSqPJ0hWyAvXv3cuDAAZKSkvDy8qJJkyYEBwfnZ20iIiIihS7X4SgtLY0hQ4bw3Xff4eDggJeXFwkJCSxbtoxmzZqxdOlSnJ2dC6JWERERkQKX68NqCxcu5NChQ8yZM4ejR4/y3Xff8dNPP/HWW2/x448/smTJkoKoU0RERKRQ5DocffrppwwbNoyOHTvi4OAAgKOjIy+88ALDhg1j27Zt+V6kiIiISGHJdTiKj48nICAgy3UBAQFcvnz5vosSERERsZVch6MqVapw6NChLNcdPHiQihUr3ndRIiIiIraS6wnZ//M//8Ps2bNxdXWlffv2lCtXjqtXr/Lpp5/y/vvvM2zYsIKoU0RERKRQ5Hrk6OWXX6Zjx47MnTuXNm3aUK9ePdq0acM777xDhw4dGDhwYK76u3btGmPGjKFZs2Y0bNiQgQMHcurUKev648eP06NHDxo0aECbNm1Ys2ZNpu3NZjMLFiwgODiYBg0aMGDAAC5cuJCpTXZ9iIiIiGTI9ciRvb09ISEh9OnTx3qdozJlyhAUFIS/v3+uCxg6dChms5lly5bh7u7Ou+++S+/evfniiy9ITU2lT58+tGnThunTp/Pjjz8yffp03N3d6dKlCwBhYWGsX7+e2bNn4+fnx9tvv03//v3Ztm0bzs7OJCQkZNuHiIiISIY8XwSyWrVqVKtW7b6ePDExkYceeohBgwZRvXp1AIYMGcLzzz/P77//zr59+3BycmLGjBk4Ojri7+/PuXPnWLZsGV26dCEtLY0VK1YwevRoWrVqBcD8+fMJDg7miy++oEOHDmzatOlv+xARERExylE4ys3NZO3s7Jg1a1aO2pYpU4Z33nnH+jg+Pp5Vq1bh5+dHtWrVWLhwIUFBQTg6/l+ZGReavHr1KhcvXuTWrVs0b97cut7T05OAgAAOHjxIhw4diIqK+ts+ypUrl+PXJiIiIsVfjsJRZGRktm0SEhJISUnJVTgy+s9//sOmTZtwdnZmyZIllCpVitjYWOuIUoYKFSoAcOnSJWJjYwHuOUOuQoUK1nXZ9XE/4cjRMV9vTVfkOTiUrNcrd+l9Lxn0PpdMet+zlqNw9NVXX/3luvT0dMLCwli2bBnlypVj2rRpeSqkV69edO3alQ8++IChQ4eyfv16UlNT77kViYuLCwC3b98mJSUFIMs2iYmJANn2kVf29nZ4ebnneXuRB4Wnp5utSxCRAqL9O2t5nnMEd88CmzBhAidOnKB9+/b85z//oUyZMnnqK2P+UkhICD/99BPr1q3D1dWVtLS0TO0yAk2pUqVwdXUF7t7vLePrjDZubnff8Oz6yCuz2UJSUnKet38QOTjYa0cqgZKSUjCZzLYuQwqY9u+SqaTt356ebjkaLctTOEpPT2fx4sW8//77lC1blkWLFvHkk0/mup/4+Hj27dvHM888Y50TZG9vT7Vq1YiLi8PPz4+4uLhM22Q89vX1JT093bqsSpUqmdrUqFEDINs+7kd6esn5QEnJZTKZ9VkXKaa0f2ct1wcbjx07RpcuXViyZAnPPvssO3bsyFMwArh69SqjRo1i37591mV37tzh2LFj+Pv7ExgYyKFDhzCZTNb1+/fvp2rVqvj4+FCzZk08PDwyzYlKSkri2LFjBAYGAmTbh4iIiIhRjsNReno6oaGhvPTSS8THx7NkyRLmzJmDp6dnnp+8evXqtGjRgpkzZ3Lw4EF+++03xo8fT1JSEr1796ZLly7cvHmTSZMmcfLkSSIiIli1ahWDBg0C7s416tGjB3PnzuXLL78kOjqakSNH4ufnR9u2bQGy7UNERETEKEeH1X799VfGjx/PyZMneeGFF5g4cSKlS5fOlwLmzZvHO++8w8iRI7lx4wZNmjThgw8+oFKlSgAsX76ckJAQOnXqRPny5Rk7diydOnWybj98+HDS09OZPHkyqampBAYGEh4ejpOTEwA+Pj7Z9iEiIiKSwc5isViya1S7dm3MZjOlS5emZs2af9+hnR2rV6/OtwKLKpPJTHz8LVuXUagcHe3x8nJnxLw9nPoj0dblSAHzf6gMoaNakZBwS3MSSgDt3yVLSd2/vb3d829CdqNGjaxfZ5elcpC1RERERIqsHIWjtWvXFnQdIiIiIkWCLo0pIiIiYqBwJCIiImKgcCQiIiJioHAkIiIiYqBwJCIiImKgcCQiIiJioHAkIiIiYqBwJCIiImKgcCQiIiJioHAkIiIiYqBwJCIiImKgcCQiIiJioHAkIiIiYqBwJCIiImKgcCQiIiJioHAkIiIiYqBwJCIiImKgcCQiIiJioHAkIiIiYqBwJCIiImKgcCQiIiJioHAkIiIiYqBwJCIiImKgcCQiIiJioHAkIiIiYqBwJCIiImKgcCQiIiJioHAkIiIiYqBwJCIiImKgcCQiIiJioHAkIiIiYmDzcHT9+nWmTJlCixYtaNSoES+//DJRUVHW9fv27aNz587Ur1+fdu3asX379kzb3759m+nTp9O8eXMaNmzIG2+8QXx8fKY22fUhIiIiksHm4WjUqFEcOXKEefPmsXnzZmrVqkW/fv04ffo0p06dYtCgQQQHBxMREcG///1vxo4dy759+6zbT5s2je+++46FCxeyevVqTp8+zfDhw63rc9KHiIiISAZHWz75uXPn+P7771m/fj2NGzcG4D//+Q/ffvst27Zt49q1a9SoUYORI0cC4O/vz7Fjx1i+fDnNmzfn8uXLbN26lffee48mTZoAMG/ePNq1a8eRI0do2LAhq1ev/ts+RERERIxsOnLk5eXFsmXLqFu3rnWZnZ0ddnZ2JCUlERUVdU+AadasGYcOHcJisXDo0CHrsgxVq1bF19eXgwcPAmTbh4iIiIiRTUeOPD09admyZaZlO3fu5Ny5c0ycOJEtW7bg5+eXaX2FChVISUkhISGBy5cv4+XlhYuLyz1tYmNjAYiNjf3bPry9vfNcv6OjzY9KFioHh5L1euUuve8lg97nkknve9ZsGo7+7PDhw0yYMIG2bdvSqlUrUlNTcXZ2ztQm43FaWhopKSn3rAdwcXHh9u3bANn2kVf29nZ4ebnneXuRB4Wnp5utSxCRAqL9O2tFJhzt3r2b0aNH06hRI+bOnQvcDTl/DjAZj93c3HB1dc0y4Ny+fRs3N7cc9ZFXZrOFpKTkPG//IHJwsNeOVAIlJaVgMpltXYYUMO3fJVNJ2789Pd1yNFpWJMLRunXrCAkJoV27dvzv//6vdWSnYsWKxMXFZWobFxdHqVKlKF26NH5+fly/fp20tLRMo0NxcXH4+vrmqI/7kZ5ecj5QUnKZTGZ91kWKKe3fWbP5wcb169fz5ptv0r17d+bNm5cp5DRp0oQDBw5kar9//34aNWqEvb09jRs3xmw2WydmA5w5c4bLly8TGBiYoz5EREREjGyaDs6cOcOsWbN4+umnGTRoEFevXuXKlStcuXKFGzdu0LNnT44ePcrcuXM5deoUK1as4PPPP6d///4A+Pr60r59eyZPnkxkZCRHjx5l1KhRBAUF0aBBA4Bs+xARERExsulhtZ07d3Lnzh127drFrl27Mq3r1KkTs2fPJiwsjLfffpvVq1fz8MMP8/bbb2c6Nf/NN99k1qxZDBs2DIAWLVowefJk6/rHH3882z5EREREMthZdLGfPDGZzMTH37J1GYXK0dEeLy93Rszbw6k/Em1djhQw/4fKEDqqFQkJtzQnoQTQ/l2ylNT929vbPUcTsjXpRkRERMRA4UhERETEQOFIRERExEDhSERERMRA4UhERETEQOFIRERExEDhSERERMRA4UhERETEQOFIRERExEDhSERERMRA4UhERETEQOFIRERExEDhSERERMRA4UhERETEQOFIRERExEDhSERERMRA4UhERETEQOFIRERExEDhSERERMRA4UhERETEQOFIRERExEDhSERERMRA4UhERETEQOFIRERExEDhSERERMRA4UhERETEQOFIRERExEDhSERERMRA4UhERETEQOFIRERExEDhSERERMRA4UhERETEQOFIRERExEDhSERERMSgSIWjpUuX0rNnz0zLjh8/To8ePWjQoAFt2rRhzZo1mdabzWYWLFhAcHAwDRo0YMCAAVy4cCFXfYiIiIhkKDLh6IMPPiA0NDTTsoSEBPr06UOVKlXYvHkzQ4cOZe7cuWzevNnaJiwsjPXr1/Pmm2+yYcMGzGYz/fv3Jy0tLcd9iIiIiGRwtHUBly9fZurUqURGRvLoo49mWrdp0yacnJyYMWMGjo6O+Pv7c+7cOZYtW0aXLl1IS0tjxYoVjB49mlatWgEwf/58goOD+eKLL+jQoUO2fYiIiIgY2Xzk6Ndff8XJyYlPPvmE+vXrZ1oXFRVFUFAQjo7/l+GaNWvG2bNnuXr1KtHR0dy6dYvmzZtb13t6ehIQEMDBgwdz1IeIiIiIkc1Hjtq0aUObNm2yXBcbG0v16tUzLatQoQIAly5dIjY2FoCKFSve0yZjXXZ9lCtXLs+1OzraPFsWKgeHkvV65S697yWD3ueSSe971mwejv5Oamoqzs7OmZa5uLgAcPv2bVJSUgCybJOYmJijPvLK3t4OLy/3PG8v8qDw9HSzdQkiUkC0f2etSIcjV1dX68TqDBmBplSpUri6ugKQlpZm/TqjjZubW476yCuz2UJSUnKet38QOTjYa0cqgZKSUjCZzLYuQwqY9u+SqaTt356ebjkaLSvS4cjPz4+4uLhMyzIe+/r6kp6ebl1WpUqVTG1q1KiRoz7uR3p6yflAScllMpn1WRcpprR/Z61IH2wMDAzk0KFDmEwm67L9+/dTtWpVfHx8qFmzJh4eHkRGRlrXJyUlcezYMQIDA3PUh4iIiIhRkQ5HXbp04ebNm0yaNImTJ08SERHBqlWrGDRoEHB3rlGPHj2YO3cuX375JdHR0YwcORI/Pz/atm2boz5EREREjIr0YTUfHx+WL19OSEgInTp1onz58owdO5ZOnTpZ2wwfPpz09HQmT55MamoqgYGBhIeH4+TklOM+RERERDLYWSwWi62LeBCZTGbi42/ZuoxC5ehoj5eXOyPm7eHUH4m2LkcKmP9DZQgd1YqEhFuak1ACaP8uWUrq/u3t7Z6jCdlF+rCaiIiISGFTOBIRERExUDgSERERMVA4EhERETFQOBIRERExUDgSERERMVA4EhERETFQOBIRERExUDgSERERMVA4EhERETFQOBIRERExUDgSERERMVA4EhERETFQOBIRERExUDgSERERMVA4EhERETFQOBIRERExUDgSERERMVA4EhERETFQOBIRERExUDgSERERMVA4EhERETFQOBIRERExUDgSERERMVA4EhERETFQOBIRERExUDgSERERMVA4EhERETFQOBIRERExUDgSERERMVA4EhERETFQOBIRERExUDgSERERMVA4EhERETEoMeHIbDazYMECgoODadCgAQMGDODChQu2LktERESKmBITjsLCwli/fj1vvvkmGzZswGw2079/f9LS0mxdmoiIiBQhJSIcpaWlsWLFCoYPH06rVq2oWbMm8+fPJzY2li+++MLW5YmIiEgRUiLCUXR0NLdu3aJ58+bWZZ6engQEBHDw4EEbViYiIiJFjaOtCygMsbGxAFSsWDHT8goVKljX5Za9vR3e3u73XduDxM7u7v/TBjQn3WS2bTFS4Bwd7v7tVKaMGxaLjYuRAqf9u2Qpqfu3vb1djtqViHCUkpICgLOzc6blLi4uJCYm5qlPOzs7HBxy9k0ubsqWdrF1CVKI7O1LxACz/H/av0sW7d9ZKxHfFVdXV4B7Jl/fvn0bNzc3W5QkIiIiRVSJCEcZh9Pi4uIyLY+Li8PX19cWJYmIiEgRVSLCUc2aNfHw8CAyMtK6LCkpiWPHjhEYGGjDykRERKSoKRFzjpydnenRowdz587F29ubhx56iLfffhs/Pz/atm1r6/JERESkCCkR4Qhg+PDhpKenM3nyZFJTUwkMDCQ8PBwnJydblyYiIiJFiJ3FUpJO4hMRERH5eyVizpGIiIhITikciYiIiBgoHImIiIgYKByJiIiIGCgciYiIiBgoHImIiIgYKByJiIiIGCgciYiIiBgoHImIiIgYKByJiIiIGCgciRQAs9ls6xJEpAD81b6tO3EVLyXmxrMihcVsNmNvf/fvjt27dxMTE0NCQgI1a9bkX//6l42rE5G8Mu7bhw4dIjY2FpPJRM2aNalevToWiwU7OzsbVyn5QTeeFSkgb7/9Ntu2baNhw4YkJSVx6NAhunTpwtSpU21dmojch7lz57Jr1y5cXV0pVaoUR44cYdmyZbRo0cLWpUk+0WE1kQKwfft2tm3bxsKFC3n33Xfp3Lkz9vb2/OMf/+DChQu2Lk9E8mjDhg1s2bKFkJAQPv74Y+tosMlkIikpycbVSX5ROBLJBxnzEDL+P336NI0aNaJ+/fp8/vnnTJ06lQkTJlCjRg3eeecdjh49astyRSSHMg6umEwmLBYLP//8M//+979p0qQJu3fvZv78+cyaNYuHH36YmTNnEhsba+OKJT8oHInkg4x5CCdPngQgISEBb29vfvjhByZMmMCYMWPo2rUrd+7cYdeuXVy8eNGW5YpIDmXMITKZTNjZ2ZGQkICPjw979uxhzJgxjB07ls6dO3PhwgW++uorrl27ZuOKJT8oHInkk927dzNixAhu3LhB06ZNWb9+PX379mXmzJm8/PLLADg5OeHv70+5cuVsXK2I5NSHH35Inz59sFgsVKpUidDQUEaOHMmECROs+7aPjw++vr64urrauFrJDwpHIvnEwcGB2NhYjh07xjPPPEPv3r1xcHAgLS2NM2fOcPbsWWbOnImbmxuNGjWydbkikgNms5lbt26RkJBAQkICY8aMoVatWpQqVYqGDRty7do14uPjWbhwIRUqVKBq1aq2Llnygc5WE8mD9PR0HB3vvRLGG2+8wfnz51m7di0JCQmsXr2adevW4e7ujo+PD2XKlGHNmjU4OTlhMplwcHCwQfUi8ley2rdv3rzJv/71L5555hkmT55MdHQ048eP5+LFi7i7u+Pl5QXAxo0bcXJyynTKvzyYFI5EcuGzzz6jZcuWlCpVCoALFy7g7e2Nu7s7AHv27CE0NJRJkyYRGBgIwC+//MKVK1coXbo0jRo1wt7e/i/DlYjYxoEDBwgKCrI+jo+Px9vb2/r4ww8/ZO3atbz99tvUqFEDgE2bNnHnzh3Kli1Lu3btcHBw0L5dTCjaiuTQggUL2LJli3VOwe7du3n66aeZNm0aX375JQDBwcF4eHgQHh5u3a5OnTq0bt2aJk2aYG9vj8lk0g9PkSJk2rRpbN261Xq26apVq+jWrRtz5szh5s2bAAQFBZGcnJzpTNOXXnqJ7t270759exwcHLRvFyMaORLJhbS0NJydnfn555+pUqUKu3bt4ttvv2XXrl089dRTdOzYET8/PyZOnMiYMWMIDg62dckiko3jx49TrVo1nJycuHjxIhaLhZUrVxIZGcn169f5n//5H1566SV27drF0qVL+eijjyhfvryuiF2MaeRIJBvHjh3j8OHDADg7O/PNN9/Qs2dPtm3bxosvvsi7777L6tWrMZlMhIaGMnz4cJKTk/npp59sXLmI/J2jR49y8+ZNatWqhZOTE1u2bKFfv36cO3eOyZMns2nTJp5//nn2799Pu3bt+Pnnn3FxcSEyMhJAwagYUzgSycYvv/zC0qVLOX36NLt376Zly5a0atWKNWvWsHbtWpKSkggMDOStt95i0aJFBAcHEx8fb/0BKiJF07fffktISAhwd/5g48aN8fT0ZMmSJXzxxRe4ubkxevRo5s6dy5tvvsnp06c5f/689u0SQIfVRLLxxRdfEBYWRnJyMjExMfz88884ODgwcuRIfv75Z1555RU6depE6dKlrducOnWKqlWr6owVkSIqPT2d9957j507d5KWlsb58+c5fvw4UVFRzJ8/H3t7e7p165bpZtHx8fGcOXOGBg0a6EzTYk4/uUWy0bZtWx599FFiYmJo3Lgxv/76KwDz58+nbt26rFmzhq1bt1onbgL4+/tbJ1+LSNHx8ssvs2XLFhwdHRk2bBheXl6cP3+eli1bAtCkSRNGjRqF2Wxm/fr17Nq1y7qtt7c3jRs3tk6+luJL4UgkCxkDqunp6QBUrVqVMWPGkJKSQlhYmHVYPSMgrVu3jg8++ICUlJRM/eivS5GiIy0tjc6dO9O+fXsArl+/jq+vL926dSM2NpZx48Zx8+ZNGjduzMiRI7FYLKxbt45t27bd05f27eJNh9VE/sR4AbeLFy/i6emJh4cHANu3byc8PJwKFSrQp08fmjZtCsCwYcMAWLhwoSZpijwAwsLC8PDw4JVXXgFg0aJF7Ny5k4CAAP7zn//g4eHBjz/+yIQJE2jWrBlTp061ccVSmHRBBhEDYzBavHgxn376KXZ2djRs2JCQkBDat2+PnZ0dy5cvZ+XKldy4cYPLly+zYMEC4O7ZKzq9V6ToMe6XaWlppKSksGDBAtLS0ujfvz8DBw7Ezs6OL774gpkzZzJq1ChcXV0JCwujSpUqNq5eCptGjkSy8Pbbb/Phhx8yaNAgEhISiIiIoE2bNsycOROAHTt2EB4eTkxMDE5OTnzzzTc4ODjotgEiRdBf7ZfvvfceoaGhvPHGGwwYMIC0tDTef/99tm3bxtmzZ6lSpQo7d+7Ezs5Ot/spYRSORP5k7969hISE8Pbbb1OvXj2++uorRo4cSenSpQkKCmLevHkAnDhxgoSEBAIDAxWMRIoo4365fft2Tp48iYODA/379wfuXg37zwHp8OHDxMTE8MILL+iK1yWU3nUp8f58GOzs2bN4eHhQr149oqOjWbduHa+//jpOTk7MmTOHKVOmMHbsWOv9lQD9VSlSRGUEo//93/9l+/btVKpUiaZNm2Jvb4+zszOvvvoqJpOJefPmYWdnR//+/WnWrJl1e90rrWTSOy4lmjEYxcXFUaFCBdzd3fH39+fSpUt8+umnVKtWja5du3LlyhXmzp3Lpk2bSE5OZu7cudZ+FIxEiq7Nmzezbds2li9fTs2aNYmLi+PkyZNERkbSsmVLhg4dir29PXPnzsXPz48OHTpYt1UwKpn0rkuJZRxu37FjB3v37mXw4MG0b9+eBg0akJ6ezp49exg5ciTu7u6cP3+eoKAgBg8eTP369W1cvYjk1KVLl+jYsSM1a9bk8OHDbNiwge+//5709HQWLlzIxx9/zKuvvkrFihVp166drcuVIkDhSEokYzD6/vvv2bp1KwcOHMDe3p6BAwfi7+/Pvn37uHXrFpUqVeLOnTssWrQIZ2dnGjZsqAmaIkVUxmiwcVT4zp07rFixghs3brBlyxZatGjB8OHDCQoKom/fvhw6dIjKlSvzwgsvADqUJgpHUkIZ5yF89tlnNG/enH/84x/s3LkTR0dHevXqxWOPPcb169cZOnQoDg4OuLm5sXnzZuzs7DCbzQpGIkWM8Y8ek8mEyWTCxcWFESNGcPv2bY4fP860adNo164dHh4e3LhxgzJlylCmTJlM/SgYic5WkxLr4MGDjB49mkWLFlG3bl3g7tks7733HnXq1GHy5MnExsby7bff4uDgwMsvv4yjo6P+qhQpgozBaPXq1ezbt4+LFy9SuXJlXnvtNWrWrAnA1atXSU9P5/bt27z11ltcuXKFTZs26Y8dyUQ/4aXESk1NxdHRkQoVKlgPkbVv3x6z2czYsWMBGDJkCL1797ZuYzKZFIxEiqCMYDR//nw+/PBDunbtyuOPP86ePXvo27cvISEhNGzYkHXr1rFs2TIef/xxSpUqxYYNG6z3SlNAkgwaOZISIat5CF988QWjRo3is88+o3Llyty+fRsXFxfu3LnDU089hclkol27drz66quUK1fOxq9ARLITHR3NqFGjmDJlivV0/PT0dAYMGMCFCxf473//S5kyZfjqq6/w9fWlXr16ODg4aDRY7qEr1kmxZzabrYHIbDaTlpYGQNu2bWnQoAFDhgwhPj4eFxcX4O7NKOvUqcMLL7zA5s2b+fnnn21Wu4jk3OXLl7l16xbVq1cH/m9i9fz580lNTeW///0vzs7OtGvXjoYNG1pHjBSM5M8UjqRYM85DWL9+PaNHj6Z79+4MHz6cU6dO0atXL5ycnPj3v//Nzp072bFjB+PHj+fatWuMHj2a2rVrs3fvXhu/ChH5s6wOejg7O3Pnzh2io6OBuxOrLRYLTk5OlCtXLssr2OtQmmRF4UiKtYwfhnPnziUsLIwaNWrQs2dPvv76a6ZNm0b9+vWZOnUqtWvXZsqUKSxevBgHBwdWrVoFgIuLC4899pgNX4GI/JlxNDg1NZVbt24BUKNGDdzd3Vm7di2//fYbcPdm0HZ2dri4uFChQgWb1SwPFo0lSrH366+/smvXLubPn09gYCDffPMNTk5OvPzyy1y5cgV/f38WLFhAbGwsTk5OuLm54eTkxIIFCzh+/DhTpkyx9UsQkf/PYrFY/+hZunQphw8f5rfffqNFixZ07dqVxYsX8+KLL5KSkkJwcDBVqlThgw8+IDU1lS5duti4enlQaORIir2rV68CEBgYyJdffsmIESMYM2YM7dq1IzQ0lCVLlgDg7e1NbGwsnTt3pmvXrmzZsoXly5fzyCOP2LJ8Efn/jCNG7733HitWrCAwMJC2bdty4sQJhg0bRmJiIhERETg4OLB8+XKWLl1KqVKl+Oijj6xzjESyo5EjKVb+fBNZAE9PTzw9PVm2bBlLlixh/PjxdO3aleTkZBITE7lz5w5wd75C7dq16d+/P76+vlSrVo2KFSva4mWISBYyRoxOnz7NqVOnmDp1Ks8++ywAv/zyC6tXr2b69Om8//77LFq0iJs3b+Lg4ICXlxd2dnY6K01yTCNHUmwY/6q8cOEC8fHxADz88MPcuHGDefPm0a9fP7p27QrcnaxpZ2dnDUBmsxmAF198keDgYAUjkSLo/PnzPPvss3z++eeZ/hCqU6cO3bp148aNGxw+fBg3NzfKly+Pt7e39TIeCkaSUwpHUiwY5yG8++67DBo0iI4dOzJ//nzc3NyYN28ezs7OHDlyhBUrVrBz504GDhxISkoKPXv2BMjyTBYRKVqqVKnCqlWrrGelpaSkWNc1bNgQLy8vfv/993u2+/OIssjf0W8DeeAZR4zef/99/vvf/9KjRw/atm3L+vXrmTNnDo8++igffPABFouFZcuWsXz5cjw8PNi8eTOOjo6ahyDyAGnWrBnh4eGEh4fz5Zdfcvv2bQBu3ryJvb09lStXtnGF8qDTFbKl2Dh//jwrV66kVatWtGzZErh7baPFixfTpk0bRo0aRZkyZbh27ZrmIYgUA3v37mXw4ME8++yzVK5cmV9//ZWLFy8SERGBk5OTrcuTB5h+I0ixcO7cOZ555hnc3Nx44oknrMu7desGQFhYGA4ODnTv3p3HH3/cul7zEEQeXC1atOD999+nb9++1K5dm9atW7N06VIA/dEj90WH1aRYeOSRR1i1ahUpKSkcOXKEhIQE67pu3boxbNgwNmzYcM/VrjUPQeTB9o9//IOVK1dy4sQJPDw8rGefKhjJ/dBhNSlWvv/+e/r378+4cePo3Lkznp6e1nW7d++mdevWul2ASDH07bffMnToUIYMGUL37t0pXbq0rUuSB5jCkRQ73377LYMGDWLs2LH3BCQAk8mkgCRSDH311VdMnDiRzz//nLJly9q6HHmAKRxJsfTdd98xePBgBg8eTK9evXB3d7d1SSJSCJKTkylVqpSty5AHnOYcSbH0xBNPEBoayvfff68flCIliPZ3yQ8aOZJiLeN2IlndVkRERCQrGjmSYk3BSEREckvhSIo9BSMREckNhSMRERERA4UjEREREQOFIxEREREDhSMRkf9PJ++KCCgcicgDqmfPntSoUSPTv5o1a9KoUSM6d+7Mxx9/nKv+Dh06xMCBA62PY2JiqFGjBhEREflduogUcbozn4g8sAICApg6dar1sclkIjY2llWrVjF27FjKli1Ly5Ytc9TXhx9+yKlTp6yPK1SowMaNG6lSpUq+1y0iRZvCkYg8sDw8PGjQoME9y1u0aEHz5s2JiIjIcTj6M2dn5yz7FpHiT4fVRKTYcXFxwdnZ2XqNq/j4eKZPn07r1q2pU6cOQUFBDB06lJiYGADGjx/Pli1b+OOPP6yH0v58WC0iIoKAgAB++uknunbtSt26dWndujXh4eGZnjsuLo6RI0cSFBREYGAgU6ZMYf78+bRp06ZwvwkikmcaORKRB5bFYiE9Pd362GQy8ccff7B48WJu3brF888/j8ViYdCgQSQmJjJ69GjKlSvHiRMnCA0NZerUqYSHhzNkyBDi4+M5duwYixYtokqVKiQnJ9/zfGazmREjRtC7d29GjBjBRx99xJw5c6hevTrBwcGkpaXRq1cvkpOTmThxIh4eHixbtozjx49Tvnz5wvzWiMh9UDgSkQfWwYMHqV27dqZldnZ2VK9enXfffZfWrVtz+fJl3NzcGDduHE2aNAGgadOmnD9/no0bNwJQpUoVvL29Mx1KyyocWSwWhgwZwr///W8AGjduzK5du9izZw/BwcF88sknnD59ms2bN1OnTh0AmjVrxlNPPVVQ3wIRKQAKRyLywKpduzbTp08H7h7OCg0N5c6dO4SGhvLYY48B4Ovry5o1a7BYLMTExHDu3DlOnz7N4cOHSUtLy/VzNmzY0Pq1s7Mz3t7e1iC1f/9+KleubA1GcHdeVOvWrYmMjLyflyoihUjhSEQeWO7u7tStW9f6uH79+nTs2JG+ffsSERGBt7c3AJ988gnz5s3j0qVLlC1bllq1auHq6pqn5/zzdvb29tbrIyUkJODj43PPNlktE5GiSxOyRaTYKFeuHFOmTOHSpUuEhIQAEBUVxbhx42jbti179+4lMjKSVatWFciZaL6+vly9evWe5deuXcv35xKRgqNwJCLFSrt27QgODubTTz/lwIEDHDlyBLPZzGuvvYavry9wd+L2Dz/8ANydZA13R4DuV1BQEDExMRw/fty6LDU1lW+//fa++xaRwqNwJCLFzsSJE3FycmLmzJnW+T8zZsxg//797Ny5kz59+hAdHQ3838RrT09Prl69yjfffENcXFyenrdDhw74+/szdOhQPv74Y77++msGDhzItWvXrJcVEJGiT+FIRIqdxx57jJ49e3LixAlOnTrFlClTOHLkCAMGDGD27NlUqlSJRYsWAXdvGwLQuXNnHnroIYYOHcrWrVvz9LyOjo6Eh4cTEBDAtGnTGDt2LI8//jhPP/00pUqVyq+XJyIFzM6iOy2KiOSL33//ndOnT9O2bdtMI0Uvvvgifn5+1kAmIkWbzlYTEcknycnJvP7663Tr1o2nn34ak8nEjh07+OWXXxg9erStyxORHNLIkYhIPvr8888JDw/n1KlTWCwWAgICGDx4ME888YStSxORHFI4EhERETHQhGwRERERA4UjEREREQOFIxEREREDhSMRERERA4UjEREREQOFIxEREREDhSMRERERA4UjEREREQOFIxERERGD/wco0LOfUyIF5gAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot the value counts as a bar plot\n",
    "df['sentiment'].value_counts().plot(kind='bar')\n",
    "\n",
    "# Set the title and labels\n",
    "plt.title('Rating Distribution')\n",
    "plt.xlabel('Rating')\n",
    "plt.ylabel('No. of Reviews')\n",
    "\n",
    "categories = df['sentiment'].value_counts().index  # Get the categories (which are the index of the value_counts Series)\n",
    "custom_labels = [cat for cat in categories]  # Create custom labels\n",
    "plt.xticks(ticks=range(len(categories)), labels=custom_labels, rotation=45)  # Set custom labels with rotation\n",
    "\n",
    "# Show the plot\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "97b599da-038a-4336-9386-3b4936875d50",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>negative</th>\n",
       "      <th>positive</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14670</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14671</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14672</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14673</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14674</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14675 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       negative  positive\n",
       "0             0         1\n",
       "1             1         0\n",
       "2             0         1\n",
       "3             0         1\n",
       "4             1         0\n",
       "...         ...       ...\n",
       "14670         0         1\n",
       "14671         0         1\n",
       "14672         0         1\n",
       "14673         0         1\n",
       "14674         0         1\n",
       "\n",
       "[14675 rows x 2 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Step 1: Create X and y\n",
    "X = df['review']\n",
    "y = pd.get_dummies(df['sentiment'])  # Step 2: One-hot encode y\n",
    "y = y.astype(int)\n",
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "b81e58b0-3cb8-476f-ae82-d188f869f6a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initial split: 80% for training, 20% for temporary dataset\n",
    "X_train, X_temp, y_train, y_temp = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "\n",
    "# Split the temporary dataset equally into validation and test sets: 10% each of the original dataset\n",
    "X_val, X_test, y_val, y_test = train_test_split(X_temp, y_temp, test_size=0.5, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "0f079d56-f258-47ab-8b69-6d1226047ffc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Chetouani\\anaconda3\\envs\\py310\\lib\\site-packages\\transformers\\tokenization_utils_base.py:2645: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Load the BERT tokenizer\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
    "\n",
    "# Tokenize and prepare the data for BERT\n",
    "def encode_reviews(tokenizer, reviews, max_length):\n",
    "    return tokenizer.batch_encode_plus(\n",
    "        reviews,\n",
    "        add_special_tokens=True,\n",
    "        return_attention_mask=True,\n",
    "        pad_to_max_length=True,\n",
    "        max_length=max_length,\n",
    "        truncation = True,\n",
    "        return_tensors='tf',\n",
    "    )\n",
    "\n",
    "# Choose a maximum sequence length for BERT\n",
    "max_length = 256\n",
    "\n",
    "# Encode the datasets\n",
    "train_encodings = encode_reviews(tokenizer, X_train.tolist(), max_length)\n",
    "val_encodings = encode_reviews(tokenizer, X_val.tolist(), max_length)\n",
    "test_encodings = encode_reviews(tokenizer, X_test.tolist(), max_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "7ba0909f-1c30-4ce1-86a2-035b4c9cbb51",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the PyTorch model were not used when initializing the TF 2.0 model TFBertModel: ['cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.bias', 'cls.predictions.bias']\n",
      "- This IS expected if you are initializing TFBertModel from a PyTorch model trained on another task or with another architecture (e.g. initializing a TFBertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing TFBertModel from a PyTorch model that you expect to be exactly identical (e.g. initializing a TFBertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "All the weights of TFBertModel were initialized from the PyTorch model.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFBertModel for predictions without further training.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      " Layer (type)                   Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      " input_ids (InputLayer)         [(None, 256)]        0           []                               \n",
      "                                                                                                  \n",
      " attention_masks (InputLayer)   [(None, 256)]        0           []                               \n",
      "                                                                                                  \n",
      " tf_bert_model (TFBertModel)    TFBaseModelOutputWi  109482240   ['input_ids[0][0]',              \n",
      "                                thPoolingAndCrossAt               'attention_masks[0][0]']        \n",
      "                                tentions(last_hidde                                               \n",
      "                                n_state=(None, 256,                                               \n",
      "                                 768),                                                            \n",
      "                                 pooler_output=(Non                                               \n",
      "                                e, 768),                                                          \n",
      "                                 past_key_values=No                                               \n",
      "                                ne, hidden_states=N                                               \n",
      "                                one, attentions=Non                                               \n",
      "                                e, cross_attentions                                               \n",
      "                                =None)                                                            \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem (Slic  (None, 768)         0           ['tf_bert_model[0][0]']          \n",
      " ingOpLambda)                                                                                     \n",
      "                                                                                                  \n",
      " dense (Dense)                  (None, 512)          393728      ['tf.__operators__.getitem[0][0]'\n",
      "                                                                 ]                                \n",
      "                                                                                                  \n",
      " dropout_37 (Dropout)           (None, 512)          0           ['dense[0][0]']                  \n",
      "                                                                                                  \n",
      " dense_1 (Dense)                (None, 256)          131328      ['dropout_37[0][0]']             \n",
      "                                                                                                  \n",
      " dense_2 (Dense)                (None, 2)            514         ['dense_1[0][0]']                \n",
      "                                                                                                  \n",
      "==================================================================================================\n",
      "Total params: 110,007,810\n",
      "Trainable params: 110,007,810\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from transformers import TFBertModel\n",
    "from tensorflow.keras.layers import Input, Dense, Dropout\n",
    "from tensorflow.keras.models import Model\n",
    "\n",
    "\n",
    "# Load the pre-trained BERT model\n",
    "bert = TFBertModel.from_pretrained('bert-base-uncased')\n",
    "\n",
    "# Build the model\n",
    "input_ids = Input(shape=(max_length,), dtype='int32', name='input_ids')\n",
    "attention_masks = Input(shape=(max_length,), dtype='int32', name='attention_masks')\n",
    "\n",
    "# Get the sequence output\n",
    "sequence_output = bert(input_ids, attention_mask=attention_masks)[0]\n",
    "\n",
    "# Select the first token's last hidden state\n",
    "cls_token = sequence_output[:, 0, :]\n",
    "\n",
    "# Add custom layers\n",
    "x = Dense(512, activation='relu')(cls_token)\n",
    "x = Dropout(0.1)(x)\n",
    "x = Dense(256, activation='relu')(x)\n",
    "output = Dense(y.shape[1], activation='softmax')(x)\n",
    "\n",
    "# Compile the model\n",
    "model = Model(inputs=[input_ids, attention_masks], outputs=output)\n",
    "\n",
    "# Adjust the learning rate\n",
    "optimizer = Adam(learning_rate=2e-5)\n",
    "\n",
    "# Compile the model with the new optimizer\n",
    "model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "# Summary of the model\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "62d06174-85dc-456c-8ed9-9624198e390d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model/bert/pooler/dense/kernel:0', 'tf_bert_model/bert/pooler/dense/bias:0'] when minimizing the loss. If you're using `model.compile()`, did you forget to provide a `loss` argument?\n",
      "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model/bert/pooler/dense/kernel:0', 'tf_bert_model/bert/pooler/dense/bias:0'] when minimizing the loss. If you're using `model.compile()`, did you forget to provide a `loss` argument?\n",
      "734/734 [==============================] - 285s 336ms/step - loss: 0.3108 - accuracy: 0.8709 - val_loss: 0.2696 - val_accuracy: 0.8950\n",
      "Epoch 2/10\n",
      "734/734 [==============================] - 231s 315ms/step - loss: 0.2337 - accuracy: 0.9126 - val_loss: 0.2860 - val_accuracy: 0.8943\n",
      "Epoch 3/10\n",
      "734/734 [==============================] - 233s 317ms/step - loss: 0.1796 - accuracy: 0.9357 - val_loss: 0.3550 - val_accuracy: 0.8793\n",
      "Epoch 4/10\n",
      "734/734 [==============================] - 228s 310ms/step - loss: 0.1374 - accuracy: 0.9530 - val_loss: 0.3625 - val_accuracy: 0.8753\n",
      "Epoch 5/10\n",
      "734/734 [==============================] - 235s 320ms/step - loss: 0.1052 - accuracy: 0.9658 - val_loss: 0.4643 - val_accuracy: 0.8766\n",
      "Epoch 6/10\n",
      "734/734 [==============================] - 237s 323ms/step - loss: 0.0849 - accuracy: 0.9717 - val_loss: 0.5497 - val_accuracy: 0.8807\n",
      "Epoch 7/10\n",
      "734/734 [==============================] - 235s 320ms/step - loss: 0.0715 - accuracy: 0.9767 - val_loss: 0.6232 - val_accuracy: 0.8793\n",
      "Epoch 8/10\n",
      "734/734 [==============================] - 233s 318ms/step - loss: 0.0687 - accuracy: 0.9768 - val_loss: 0.6031 - val_accuracy: 0.8787\n",
      "Epoch 9/10\n",
      "734/734 [==============================] - 228s 310ms/step - loss: 0.0594 - accuracy: 0.9804 - val_loss: 0.6049 - val_accuracy: 0.8807\n",
      "Epoch 10/10\n",
      "734/734 [==============================] - 226s 308ms/step - loss: 0.0630 - accuracy: 0.9790 - val_loss: 0.5987 - val_accuracy: 0.8800\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.keras.callbacks import ModelCheckpoint\n",
    "\n",
    "model_save_path = r\"D:\\Marketing Paper\\best_model_k8.h5\"\n",
    "\n",
    "checkpoint = ModelCheckpoint(model_save_path, monitor='val_accuracy', save_best_only=True)\n",
    "\n",
    "# Train the model\n",
    "history = model.fit(\n",
    "    [train_encodings['input_ids'], train_encodings['attention_mask']],\n",
    "    y_train,\n",
    "    validation_data=([\n",
    "        val_encodings['input_ids'], val_encodings['attention_mask']\n",
    "    ], y_val),\n",
    "    epochs=10,\n",
    "    batch_size=16,  # Specify your desired batch size here\n",
    "    callbacks=[checkpoint]\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "95abedb1-f3ee-4228-901d-6158d081d10f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Error in loading the saved optimizer state. As a result, your model is starting with a freshly initialized optimizer.\n",
      "46/46 [==============================] - 17s 189ms/step - loss: 0.2663 - accuracy: 0.9033\n",
      "Test Loss: 0.26634666323661804, Test Accuracy: 0.9032697677612305\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.keras.models import load_model\n",
    "\n",
    "# Load the best model\n",
    "best_model = load_model(model_save_path, custom_objects={'TFBertModel': TFBertModel})\n",
    "\n",
    "# # Recompile the model with the optimizer, loss, and metrics\n",
    "# model.compile(optimizer=Adam(learning_rate=2e-5), loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "# Evaluate the model on the test set\n",
    "test_loss, test_acc = best_model.evaluate(\n",
    "    [test_encodings['input_ids'], test_encodings['attention_mask']],\n",
    "    y_test\n",
    ")\n",
    "\n",
    "print(f\"Test Loss: {test_loss}, Test Accuracy: {test_acc}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "6c2ebb00-c187-4382-bbb7-0e510c9f3df9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "46/46 [==============================] - 15s 189ms/step\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "    negative       0.89      0.92      0.91       766\n",
      "    positive       0.91      0.88      0.90       702\n",
      "\n",
      "    accuracy                           0.90      1468\n",
      "   macro avg       0.90      0.90      0.90      1468\n",
      "weighted avg       0.90      0.90      0.90      1468\n",
      "\n",
      "Accuracy for class negative: 0.92\n",
      "Accuracy for class positive: 0.88\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "\n",
    "# Assuming `test_encodings` is already prepared similar to `train_encodings` and `val_encodings`\n",
    "predictions = best_model.predict([test_encodings['input_ids'], test_encodings['attention_mask']])\n",
    "\n",
    "# Assuming you have predictions from the model\n",
    "predicted_class_indices = predictions.argmax(axis=1)\n",
    "\n",
    "class_names = y_test.columns.tolist() \n",
    "\n",
    "# Convert y_test from one-hot encoding to class indices\n",
    "true_class_indices = y_test.idxmax(axis=1).apply(class_names.index).values\n",
    "\n",
    "\n",
    "# Generate the classification report using the true and predicted class indices\n",
    "report = classification_report(true_class_indices, predicted_class_indices, target_names=class_names)\n",
    "print(report)\n",
    "\n",
    "# Generate the confusion matrix\n",
    "cm = confusion_matrix(true_class_indices, predicted_class_indices)\n",
    "\n",
    "# Calculate accuracy for each class\n",
    "class_accuracies = cm.diagonal() / cm.sum(axis=1)\n",
    "for class_name, accuracy in zip(class_names, class_accuracies):\n",
    "    print(f'Accuracy for class {class_name}: {accuracy:.2f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b860e51-c4dc-42c8-81fb-fa207fa02a26",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
